﻿import json
import math
import os
import time
from urllib import parse

import facade
import requests
from xjlibrary.our_file_dir import BaseDir
from xjlibrary.tools.BaseUrl import BaseUrl

curPath = BaseDir.get_file_dir_absolute(__file__)
TopPath = BaseDir.get_upper_dir(curPath, -2)
dirPath = BaseDir.get_new_path(TopPath, "download", "EI", "download", "json", "coden")


class DownIndexByCodenYear(object):
    def __init__(self):
        self.configfile = BaseDir.get_new_path(curPath, "db.ini")
        self.logger = facade.get_streamlogger()
        self.mysqlutils = facade.MysqlUtiles(self.configfile, "db", logger=self.logger)
        self.code_list = []
        self.year_floor = None
        self.year_ceil = None
        self.Proxies = {
            "http": "192.168.30.176:8207",
            "https": "192.168.30.176:8207"  # key是指目标网站的协议
        }
        self.sn = requests.Session()
        self.UserAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36'
        self.sn.headers['User-Agent'] = self.UserAgent
        self.NumPerPage = 100  # 检索结果页每页25条
        self.NumAllPage = 5200  # 最大条数限制
        self.init_form()

    def init_form(self):
        self.headers = {
            "Accept": "application/json, text/javascript, */*; q=0.01",
            "Accept-Encoding": "gzip, deflate, br",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Connection": "keep-alive",
            "Content-Type": "application/json",
            "Host": "www.engineeringvillage.com",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36",
            "X-NewRelic-ID": "VQQAUldRCRAFUFFQBwgCUQ==",
            "X-Requested-With": "XMLHttpRequest"
        }
        self.t = int(round(time.time() * 1000))
        self.Form = {
            "usageOrigin": "searchform",
            "usageZone": "quicksearch",
            "editSearch": "",
            "isFullJsonResult": "true",
            "angularReq": "true",
            "CID": "searchSubmit",
            "searchtype": "Quick",
            "origin": "searchform",
            "category": "quicksearch",
            "section1": "CN",
            "searchWord1": "anhui medical university",
            "allDb": "1",
            "database": "1",
            "yearselect": "yearrange",
            "startYear": "1884",
            "endYear": "2018",
            "updatesNo": "1",
            "language": "NO-LIMIT",
            "doctype": "NO-LIMIT",
            "sort": "yr",
            "treatmentType": "NO-LIMIT",
            "searchStartTimestamp": str(int(round(time.time() * 1000))),
            "_": str(self.t)
        }

        self.FormPage100 = {
            "pageSizeVal": "100",
            "SEARCHID": "",
            "sortsort": "relevance",
            "sortdir": "dw",
            "angularReq": "true",
            "isFullJsonResult": "false",
            "usageOrigin": "searchresults",
            "usageZone": "resultsperpagebottom",
            "_": ""
        }

    def init_coden_list(self):
        """
        获取coden
        :return:
        """
        self.logger.info('InitCodeList ...')
        sql = "SELECT DISTINCT `coden` FROM coden;"
        rows = self.mysqlutils.SelectFromDB(sql)
        for row in rows:
            organ = row['coden']
            self.code_list.append(organ)
        self.logger.info('code_list size:%d' % len(self.code_list))

    def set_year(self):
        # 确定年份
        curTime = time.localtime()
        if curTime.tm_mon < 4:  # 前一年和当前年
            self.year_floor = curTime.tm_year - 1
            self.year_ceil = curTime.tm_year + 1
        elif curTime.tm_mon > 9:  # 当前年和后一年
            self.year_floor = curTime.tm_year
            self.year_ceil = curTime.tm_year + 2
        else:  # 当前年
            self.year_floor = curTime.tm_year
            self.year_ceil = curTime.tm_year + 1

    def down_all(self):
        for i in range(0, 3):  # 重复三轮
            for year in range(self.year_floor, self.year_ceil):
                for coden in self.code_list:
                    self.down_one(year, coden)

        self.logger.info('Game Over! Good Boy!')

    def down_one(self, year, coden):
        self.logger.info('DownOneCoden %d, %s, %s ...' % (year, coden, repr(self.Proxies)))
        url = r'https://www.engineeringvillage.com/'
        BoolResult, errString, r = facade.BaseRequest(url,
                                                      sn=self.sn,
                                                      proxies=self.Proxies,
                                                      timeout=60)
        self.logger.info(errString)
        if not BoolResult:
            return False
        self.logger.info("首页访问完成")

        form = self.Form
        form['searchWord1'] = coden
        form['startYear'] = str(year)
        form['endYear'] = str(year)
        url = r'https://www.engineeringvillage.com/search/submit.url'

        url = url + "?" + BaseUrl.dicts_to_url(form)

        BoolResult, errString, r = facade.BaseRequest(url=url,
                                                      sn=self.sn,
                                                      proxies=self.Proxies,
                                                      allow_redirects=True,
                                                      endstring="",
                                                      timeout=60)
        self.logger.info(errString)
        if not BoolResult:
            return False
        self.logger.info("搜索成功")

        param_dict = parse.parse_qs(parse.urlparse(r.url).query)
        SEARCHID = param_dict['SEARCHID'][0]

        if not json.loads(r.text)["results"]:
            self.logger.info("没有 results 退出")
            return

        # 获取搜索出来的总量
        searchnum = json.loads(r.text)["pagenav"]["resultscount"]
        pagecount = json.loads(r.text)["pagenav"]["pagecount"]
        self.logger.info("25: 第一次搜索每页25的数量和页数:总量为{},页数为{}".format(searchnum, pagecount))
        self.logger.info("获取数据长度{}".format(len(json.loads(r.text)["results"])))
        outDir = os.path.join(dirPath, str(year), coden)
        if not os.path.exists(outDir):
            os.makedirs(outDir)

        if len(json.loads(r.text)["results"]) < 25:
            pathfile = os.path.join(outDir, str(1) + '.json')
            BaseDir.single_write_wb_file(r.content, pathfile)

            if pagecount != 1:
                self.logger.info("err : pagecount is :{}".format(pagecount))
                self.logger.info(
                    "year:{}, coden: {}, searchnum: {}, pagecount: {}, but at one search results number < 25 over ".format(
                        year, coden, searchnum, pagecount))
            return

        # 将每页转换成100
        self.FormPage100["SEARCHID"] = SEARCHID
        self.FormPage100["_"] = str(self.t)
        url = "https://www.engineeringvillage.com/search/results/quick.url"
        url = url + "?" + BaseUrl.dicts_to_url(self.FormPage100)
        BoolResult, errString, r = facade.BaseRequest(url=url,
                                                      sn=self.sn,
                                                      proxies=self.Proxies,
                                                      headers=self.headers,
                                                      allow_redirects=True,
                                                      endstring="",
                                                      timeout=60)
        self.logger.info(errString)
        if not BoolResult:
            return False

        searchnum = json.loads(r.text)["pagenav"]["resultscount"]
        pagecount = json.loads(r.text)["pagenav"]["pagecount"]
        self.logger.info("100: 转化搜索每页100的数量和页数:总量为{},页数为{}".format(searchnum, pagecount))
        pathfile = os.path.join(outDir, str(1) + '.json')
        BaseDir.single_write_wb_file(r.content, pathfile)

        if len(json.loads(r.text)["results"]) < 100:
            if pagecount != 1:
                self.logger.info("err : pagecount is :{}".format(pagecount))
                self.logger.info(
                    "year:{}, coden: {}, searchnum: {}, pagecount: {}, but at 100 page results number < 100 over ".format(
                        year, coden, searchnum, pagecount))
            return

        if pagecount > int(math.ceil(self.NumAllPage / self.NumPerPage)):
            pagecount = math.ceil(self.NumAllPage / self.NumPerPage)  # 假定最大页码

        for PAGE in range(2, int(pagecount) + 1):  #
            pathfile = os.path.join(outDir, str(PAGE) + '.json')
            if os.path.exists(pathfile):  # 该页已经存在
                print('exists ' + pathfile + ' ...')
                continue

            if not self.DownOnePage(outDir, SEARCHID, pagecount, PAGE):  # 下载失败，不再继续本次会话
                if pagecount != PAGE:
                    self.logger.info('Down ' + str(PAGE) + ' failed')
                    self.logger.info(
                        "year:{}, coden: {}, searchnum: {}, pagecount: {}, at {} page failed,should down {} page ".format(
                            year, coden, searchnum, pagecount, PAGE, pagecount))
                    if 'sn' in locals() and self.sn:
                        self.sn = None
                break

        return True

    def DownOnePage(self, outDir, SEARCHID, pagecount, PAGE):
        self.logger.info('DownOnePage ' + str(PAGE) + '/' + str(pagecount) + ' ...')

        BaseUrls = "https://www.engineeringvillage.com/search/results/quick.url"
        FormPagenumber = {
            "navigator": "NEXT",
            "SEARCHID": SEARCHID,
            "database": "1",
            "angularReq": "true",
            "isFullJsonResult": "false",
            "usageOrigin": "searchresults",
            "COUNT": str((int(PAGE) - 1) * 100 + 1),
            "usageZone": "nextpage",
            "_": str(self.t)
        }

        url = BaseUrls + "?" + BaseUrl.dicts_to_url(FormPagenumber)

        BoolResult, errString, r = facade.BaseRequest(url=url,
                                                      sn=self.sn,
                                                      proxies=self.Proxies,
                                                      endstring="",
                                                      timeout=60)
        self.logger.info(errString)
        if not BoolResult:
            return False

        # 写入文件
        outPathFile = os.path.join(outDir, str(PAGE) + '.json')
        self.logger.info('outPathFile:' + outPathFile)
        BaseDir.single_write_wb_file(r.content, outPathFile)
        self.logger.info("获取数据长度{}".format(len(json.loads(r.text)["results"])))
        if len(json.loads(r.text)["results"]) < 100:
            return False

        return True

    def run(self):
        self.init_coden_list()
        self.set_year()
        self.down_all()


if __name__ == "__name__":
    down = DownIndexByCodenYear()
    down.run()

